package ca.uwindsor.cs.deepweb.utility.sougouweb;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Date;
import java.util.HashSet;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.LockObtainFailedException;
import org.xml.sax.InputSource;

import ca.uwindsor.cs.deepweb.estimation.FilesContentIndexer;

public class DocumentSizeIndexerforSogou {

	private int globalID;

	protected IndexWriter indexWriter;

	private SogouWebCorpusDocumentHandler handler;

	private InputStreamReader reader;

	private HashSet<String> indexdocs;

	private int numDocs;

	/**
	 * Constructor
	 */
	public DocumentSizeIndexerforSogou() {
		globalID = 0;
	}

	/**
	 * Start to create the index to store file size
	 * 
	 * @param indexdir
	 *            Where to store the index
	 * @param sizeindexdir 
	 * @param docdir
	 *            Where are the documents locate
	 */
	public void index(String indexdir, String sizeindexdir, File docdir) {
		try {
			IndexReader indexreader = IndexReader.open(indexdir);
			numDocs = indexreader.numDocs();
			indexdocs = new HashSet<String>(numDocs);
			String id;

			for (int i = 0; i < numDocs; i++) {
				id = indexreader.document(i).get(FilesContentIndexer.FIELD_ID);
				indexdocs.add(id);
			}
			indexreader.close();

			Analyzer luceneAnalyzer = new StandardAnalyzer();
			long startTime = new Date().getTime();
			indexWriter = new IndexWriter(sizeindexdir, luceneAnalyzer, true);
			index(docdir);

			long endTime = new Date().getTime();
			System.out
					.println("It takes "
							+ (endTime - startTime)
							+ " milliseconds to create index for the files in directory "
							+ docdir.getPath()
							+ "\nTotal documnents indexed is:" + globalID);

			System.out
					.println("index is completed! \n Start to optimize the index...");
			indexWriter.optimize();
			System.out.println("optimization is completed!");
			indexWriter.close();
		} catch (CorruptIndexException e) {
			e.printStackTrace();
			System.exit(1);
		} catch (LockObtainFailedException e) {
			e.printStackTrace();
			System.exit(1);
		} catch (IOException e) {
			e.printStackTrace();
			System.exit(1);
		}

	}

	protected void index(File dir) throws CorruptIndexException,
			LockObtainFailedException, IOException {
		File[] dataFiles = dir.listFiles();

		// Add documents to the index
		for (int i = 0; i < dataFiles.length; i++) {
			File file = dataFiles[i];
			if (file.isFile()) {
				// make the ID from the path shorter - get the file name
				String filepath = file.getName();
				int offset1 = filepath.lastIndexOf(java.io.File.separator);
				int offset2 = filepath.lastIndexOf(".");
				String word = new String(filepath.substring(offset1 + 1,
						offset2));
				if (file.getName().endsWith(".xml") && indexdocs.contains(word)) {
					handler = new SogouWebCorpusDocumentHandler();
					try {
						reader = new InputStreamReader(
								new FileInputStream(file), "UTF-8");

						handler.processDocument(new InputSource(reader));
					} catch (Exception e) {
						System.err
								.println("Error on parsing: " + i + " file at "
										+ dataFiles[i].getCanonicalPath());
						continue;
					}

					String content = handler.getContent().replaceAll("+", "");

					System.out.println("Indexing the length "
							+ content.length() + "of file "
							+ dataFiles[i].getCanonicalPath());

					Document document = new Document();

					Field FieldPath = new Field(FilesContentIndexer.FIELD_ID,
							word, Field.Store.YES, Field.Index.UN_TOKENIZED);

					Field FieldSize = new Field(FilesContentIndexer.FIELD_SIZE,
							Integer.toString(content.length()),
							Field.Store.YES, Field.Index.UN_TOKENIZED);

					document.add(FieldPath);
					document.add(FieldSize);
					indexWriter.addDocument(document);
					// txtReader.close();
					globalID++;
				}
			} else if (file.isDirectory()) {
				index(dataFiles[i]);
			}
			if (globalID == numDocs)
				break;
		}

	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		if (args.length == 3) {
			DocumentSizeIndexerforSogou index = new DocumentSizeIndexerforSogou();
			index.index(args[0], args[1], new File(args[2]));
		} else {
			System.out
					.println("usage: java -jar DocumentSizeIndexerforGOV2 documentDir indexDir");
		}
	}

}
